#!/usr/bin/env python3
# -*- coding:utf-8 -*-
# @Time: 2019/5/31  17:07
# @Author: 余浪人
# @Email:yulangren520@Gmail.com

# https://www.cnblogs.com/mafeng/p/5651323.html  架构图
from apps.spiders.crawler import download_get, queue, distinct_url_verify, strReplace, conversion_code
from lxml import etree
import re


def xpath_analysis_list(list_page_url: str, obj):
    response = download_get(list_page_url,obj.agreement_cls)
    if response:
        html = etree.HTML(conversion_code(response))
        dataList = html.xpath(f'{obj.list_rule}')
        if len(dataList):
            for data in dataList:
                try:
                    detail_url = data.xpath(f'.{obj.URL_rule}')[0]  # 解析出 详细页面URl
                except:
                    continue
                if queue.full(): return
                if not distinct_url_verify(detail_url):  # 链接去重
                    queue.put({"detail_url": detail_url, "obj": obj})
            return
    return "列表解析为空"


def re_analysis_list(list_page_url: str, obj):
    response = download_get(list_page_url,obj.agreement_cls)
    if response.text:
        response = conversion_code(response)
        try:
            s, e = strReplace(obj.list_rule)
            List = re.search(f'{s}([\s\S]*?){e}', response, re.S).group(1)  # 列表区域
        except:
            return "列表规则错误"
        if List:
            s, e = strReplace(obj.URL_rule)
            urlList = re.findall(f'{s}(.*?){e}', List)
            if len(urlList):
                for detail_url in urlList:
                    if not distinct_url_verify(detail_url):  # 链接去重
                        if obj.url_relative:
                            url = obj.goal_url + detail_url
                            queue.put({"detail_url": url, "obj": obj})
                        else:
                            queue.put({"detail_url": detail_url, "obj": obj})
                return urlList
            return "列表链接解析为空"
    return "列表解析为空"


if __name__ == '__main__':
    list_url = 'http://www.zzlzl.com/sort/1'
